#!/bin/sh
#Create By Xuezhang@2012-07-05
#For Server OPS Check
warnum=0
rm -rf /tmp/ops_check_record.log
touch /tmp/ops_check_record.log
host=`hostname`
MYSQL="/tmp/mysql -h172.19.70.68 --default-character-set=gbk -umonitor -ptb4monitor"
#All warning write into tmp.log
set_env1(){
	kers=`uname -r`
	rm -rf /usr/bin/perf
	ln -s  /usr/libexec/perf.${kers} /usr/bin/perf

}
getwarn()
{
	warnum=`expr $warnum + 1`
	echo "$1     " >> /tmp/ops_check_record.log
}

#At Last Put Warning in summary
putwarn()
{  

	if [ $warnum  -eq 0 ];then echo "This Service is very NICE,Just Use it";
	else
		echo  "This Service have $warnum Warnings;Please Check Following"
		while read line
		do
			echo "$line"
		done <   /tmp/ops_check_record.log
	fi
	mv /tmp/ops_check_record.log  /tmp/ops_check_record_bak.Check
}



#log raid config;If fount Raid0,exit immediately
check_megaraid()
{
	raiderr=`cat  /tmp/ops_his_raid.log|grep -i 'Primary-0'|wc -l`
	if [ $raiderr -gt 0 ];then echo "RAID LEVEL ERROR";$warnum=99;fi
	cat /tmp/ops_his_disk.log|grep -E "(Raw Size)|(Virtual Drive)" | awk '{if($0 ~ /Virtual Drive/){printf("\n%s","vd"$3 "\n");b="$0"}else{printf("%d ",$3 ,"\n")}}' > /tmp/ops_his_disk2.log
	tun=`cat /tmp/ops_his_disk2.log|grep vd|wc -l`
	for ((i=0;i<$tun;i++));do
		num=`cat  /tmp/ops_his_disk2.log|grep vd${i} -A 1|awk '{for(i=1;i<NF;i++){a[$i]++}}END{for(l in a)printf l":"a[l]",\n"}'|wc -l`
		if [ $num -gt 1 ];then
			getwarn "DIFF DISK vd${i} in ONE RAID"
		fi
	done
	SSDv=`cat /tmp/ops_his_disk.log|grep -i ssd|wc -l`
	FIOv=`/sbin/lspci  |grep -i Fusion-io|wc -l`
	SASnum=`cat /tmp/ops_his_disk.log|grep 'Device Id'|wc -l`
	SASize=`cat /tmp/ops_his_disk.log|grep 'Raw Size'|awk {'print $3'}|tail -n 1`
	Plocyv=`cat  /tmp/ops_his_raid.log|grep 'Default Cache Policy'|tail -n 1 |awk  '{print $4}'`
	cachev=`cat  /tmp/ops_his_raid.log|grep 'Disk Cache Policy'|grep -i 'Enabled'|wc -l`
	[ $cachev -gt 0  ] &&  getwarn "DISK cache is enable"
}
chekc_hpraid(){
	echo "this is hp raid server"
	tmpslot=`hpacucli ctrl all show status | grep 'Smart Array'|awk '{ print $6 }'`
	raiderr=`hpacucli ctrl slot=3 ld all show status|grep -i 'raid 0'|wc -l`
	if [ $raiderr -gt 0 ];then echo "RAID LEVEL ERROR";$warnum=99;fi
	hpacucli ctrl slot=${tmpslot} pd all show status > /tmp/ops_his_disk.log
	SASize=`cat /tmp/ops_his_disk.log|awk '{print $7}'|uniq |sort|tail -n 1`
	SASnum=`cat /tmp/ops_his_disk.log|grep ${SASize}|wc -l`
	FIOv=`/sbin/lspci  |grep -i Fusion-io|wc -l`
	SSDv=`cat /tmp/ops_his_disk.log|grep -i ssd|wc -l`
	#SSDv= 
	#FIOv=
	#
	#Plocyv=

	#hpacucli ctrl slot=${tmpslot} ld all show status
	#hpacucli ctrl slot=${tmpslot} show config detail 
}

#check which ?¨²t is 
check_Dx()
{
	kernelv=`uname -r|grep tb|grep -v grep|wc -l`
	memg=`free -g|grep Mem|awk '{print $2}'`
	if [ $kernelv -eq 1 ] ;then
		echo "TAOBAO-Kernel Server"
	else
		echo "NOT taobao-kernel Server"
	fi
	if [ ${SSDv} -gt 4 ] ;then
		echo "Disk SSD $SASize *$SSDv $Plocyv;MEM $memg"
	elif [ ${FIOv} -ge 1 ] ;then
		echo "FusionIO + SAS $SASize *$SASnum $Plocyv;MEM $memg"
	else
		echo "Disk SAS ${SASize}GB *$SASnum $Plocyv;MEM $memg"
	fi
	if [ $SASnum -gt 13 ] ;then
		getwarn "Disk num is too much,sth wrong"
	fi 
	#SQL
}


#Check Base rpm,support for OS/Dbagent
check_rpm()
{
	if [ `cat /tmp/ops_his_rpm.log|grep $1|wc -l` -lt 1  ];then
		getwarn "$1 is not install"
	fi
}

check_bond()
{
	[  ! -e /proc/net/bonding/bond0 ] && getwarn  "No bond on $host " 
	crdevc=`cat /proc/net/bonding/bond0|grep 'Active Slave'|awk -F":" '{print $2}'`
	devc=($(cat /proc/net/bonding/bond0 |grep Interface|awk -F":" '{print $2}'|tr '\n' " "))
	if [ ${devc[0]} = $crdevc ] ; then
		ifenslave -c bond0 ${devc[1]}
	elif [ ${devc[1]} = $crdevc  ];then
		ifenslave -c bond0 ${devc[0]}
	fi
	sleep 2
	ping dbconsole.cm3 -c 3 > /tmp/pingdb.log
	ifenslave -c bond0 $crdevc
	if [ $? -gt 0 ] ;then
		echo "$host bond rollback wrong"
	fi
	num=`cat /tmp/pingdb.log|grep '3 received'|wc -l`
	if [ $num -lt 1 ];then
		getwarn  "$host ping test wrong"
	fi
}
#Check DBA USERS
check_user()
{
	if [ ! -d "/home/$1" ] ;then
		getwarn "$1 user not exist"
	fi
}
#Check ulimit
check_limit()
{
	ulimit -a > /tmp/ops_his_ulimit.log
	num=`cat /tmp/ops_his_ulimit.log|grep 'max user processes'|awk -F ")" '{print $2}'`
	if [ $num -lt 131070  ];then
		getwarn "ulimit max user processes ERROR"
	fi
	num=`cat /tmp/ops_his_ulimit.log|grep 'max locked memory'|awk -F ")" '{print $2}'`
	if [ $num -lt 64   ];then
		getwarn "ulimit max locked memory ERROR"
	fi
	# num=`cat /tmp/ops_his_ulimit.log|grep 'file locks'|awk -F ")" '{print $2}'`
	# if [ $num = "unlimited"   ];then
	#    getwarn "ulimit file locks ERROR"
	#    fi
	num=`cat /tmp/ops_his_ulimit.log|grep 'open files'|awk -F ")" '{print $2}'`
	if [ $num -lt 131070  ];then
		getwarn "ulimit 131070 ERROR"
	fi   
}

#Check grub.conf for numa and schedule
num=`cat /boot/grub/grub.conf |grep title -A 2|head -n 3|grep 'numa=off'|wc -l`
if [ $num -lt 1   ];then
	getwarn "numa ERROR"
fi
num=`numastat |grep -i node1|wc -l`
if [ $num -gt 0   ];then
	getwarn "numa ERROR"
fi
num=`cat /boot/grub/grub.conf |grep title -A 2|head -n 3|grep 'elevator=deadline'|wc -l`
if [ $num -lt 1   ];then
	getwarn "scheduler ERROR"
fi

#check Netoops for tb-kernel
if [ ! -f /sys/kernel/config/netoops/target1/enabled  ];then 
	getwarn "netoops is not install"
else 
	num=`cat /sys/kernel/config/netoops/target1/enabled `
	if [ $num -lt 1   ];then
		getwarn "netoops is not start"
	fi
fi

num=`ps -ef|grep crond |grep -v grep|wc -l`
if [ $num -lt 1 ];then
	getwarn "crontab is not start"
fi
num=`/bin/mount |grep ext |wc -l`
if [ $num -lt 7 ];then
	getwarn "Partition num is not enough"
fi

if [ ! -d /u01 ];then
	getwarn "u01 dir is not exist"
fi

num=`cat /etc/ssh/sshd_config |grep -i 'UseDNS no'|wc -l`
if [ $num -lt 1 ];then
	getwarn "ssh enable DNS"
fi


rpm -qa > /tmp/ops_his_rpm.log
check_rpm "kernel-devel"
check_rpm "kernel-debuginfo"
check_rpm "kernel-debuginfo-common"
check_rpm "telnet"
check_rpm "nmap"
check_rpm "sysstat"
check_rpm "ksh"
check_rpm "syslinux"
check_rpm "screen"
#####################
check_user "mysql"
check_user "jiyuan"
check_user "supu"
check_user "tianyu"
check_user "liukun"
check_user "mugong.zjq"
check_user "yuanyun.wk"
check_user "rulong"
check_user "suzhen"


u01space=`df |grep u01|awk {'printf  $2/1024/1024'}|xargs printf "%.f"`
if [ `expr ${u01space} - 1300` -lt 0 ] ;then
	#	getwarn "u01 space is too small"
	echo "u01 space is too small"
fi

if [ -d /u02 ] ;then 
	u02space=`df |grep u02|awk {'printf  $2/1024/1024'}|xargs printf "%.f"`
	if [ `expr ${u02space} - 400` -lt 0 ] ;then
		#getwarn "u02 space is too small"
		echo "u02 space is too small"
	fi
fi
check_limit;

megav=`/sbin/lspci  |grep -i Megaraid|wc -l`
hpraidv=`/sbin/lspci  |grep -i RAID|grep -i Hewlett|wc -l`

if [ $megav -ge 1 ] ;then
	MegaCli=`rpm -qa|grep -i MegaCli|wc -l`
	if [ $MegaCli -eq 0  ] ;then
		yum install -y MegaCli.i386
	fi
	/opt/MegaRAID/MegaCli/MegaCli64 -LDSetProp DisDskCache -lall -a0
	/opt/MegaRAID/MegaCli/MegaCli64  -PDList  -aAll -NoLog  > /tmp/ops_his_disk.log
	/opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -Lall -aALL -NoLog > /tmp/ops_his_raid.log 
	check_megaraid;
elif [[ hpraidv -ge 1 ]]; then
	hpacucliv=`rpm -qa|grep -i hpacucli|wc -l`
	if [ $hpacucliv -eq 0  ] ;then
		yum install -y hpacucli
	fi
	chekc_hpraid
fi
check_bond;
check_Dx;
set_env1;
putwarn;

#warnum=0   #for test
#if [ $raiderr  -eq 0   ]  &&  [ $warnum  -eq 0 ];then
[ -e /opt/MegaRAID/MegaCli/MegaCli64 ] && disk_num=` /opt/MegaRAID/MegaCli/MegaCli64 -PDList -a0|grep 'Device Id'|wc -l`
if [ $disk_num -eq 0 ]  ;then
	disk_num=`cat /tmp/ops_his_disk.log|grep physicaldrive|wc -l`
fi
[ ! $disk_num ]  && disk_num=0
cpu_num=`cat /proc/cpuinfo |grep processor|wc -l`
mem_num=`free -m|grep Mem|awk '{print $2}'`
os_release=`cat /etc/issue|grep release|awk '{print $7}'`
kernel_ver=`uname -a|awk '{print $3}'`
u01space=` df -ml /u01 | grep "%" | tail -n 1 | awk '{print ($(NF-4)/1024)}'|awk -F"." '{print $1}'`
if [  $SSDv -gt 0  ];then 
	TAG=ssd
elif [  $FIOv  -gt 0 ];then
	TAG=fio
else
	TAG=''
fi
if [ $raiderr  -gt 0   ] ||  [ $warnum  -gt 0 ];then
	TAG="Wrong"
else 
	TAG="ced $1 $TAG"
fi  
echo $TAG
